# #!/usr/bin/env python
# # -*- encoding: utf-8 -*-
# '''
# @File    :   country_asn.py
# @Contact :   291622538@qq.com
#
# @Modify Time      @Author    @Version    @Description
# ------------      -------    --------    -----------
# 2020/10/25 17:00   fan        1.0         None
# '''
# from scrapy import Spider, Request
# from fan_project.items import *
# from scrapy_redis.spiders import RedisSpider
#
#
# class CountryASN(RedisSpider):
#     # 爬虫名称
#     name = "country_asn_spider_redis"
#     # 专用的通道
#     custom_settings = {
#         'ITEM_PIPELINES': {'fan_project.pipelines.SaveCountryASNRedisPipeline': 301},
#     }
#
#     # 爬虫入口
#     def start_requests(self):
#
#         URL = "https://whois.ipip.net/countries"
#         # dont_filter = False 相同的url只请求一次
#         yield Request(URL, callback=self.parse)
#
#     def parse(self, response, **kwargs):
#         country_asn_list = response.xpath('//table/tr')
#         # print(len(country_asn_list))
#         for num, country_asn in enumerate(country_asn_list, 1):
#             if num > 1:
#                 break
#
#             td = country_asn.xpath('td')
#             countryName = "".join(td[0].xpath('a/text()').extract())
#             countryHref = "".join(td[0].xpath('a/@href').extract())
#             countryCode = countryHref.split("/")[-1]
#             allocatedASNs = "".join(td[1].xpath('./text()').extract())
#             announcedASNs = "".join(td[2].xpath('./text()').extract())
#             asnIpv4Number = "".join(td[3].xpath('./text()').extract())
#             asnIpv6Number = "".join(td[4].xpath('./text()').extract())
#             countryHref = response.urljoin(countryHref)
#             countryASNItem = CountryASNItem(countryName=countryName, countryHref=countryHref, countryCode=countryCode,
#                                             allocatedASNs=allocatedASNs,
#                                             announcedASNs=announcedASNs, asnIpv4Number=asnIpv4Number,
#                                             asnIpv6Number=asnIpv6Number)
#
#             yield countryASNItem
#             yield Request(countryHref, meta={"countryCode": countryCode}, callback=self.asn_list_parse,
#                           dont_filter=False)
#
#     def asn_list_parse(self, response):
#         asn_list = response.xpath('//table/tr')
#         # print(len(asn_list))
#         for num, country_asn in enumerate(asn_list, 1):
#             if num > 10:
#                 break
#
#             td = country_asn.xpath('td')
#             asnNum = "".join(td[0].xpath('a/text()').extract())
#             asnNumHref = "".join(td[0].xpath('a/@href').extract())
#             asnName = "".join(td[1].xpath('./text()').extract())
#             ipv4NumIps = "".join(td[2].xpath('./text()').extract())
#             ipv6NumIps = "".join(td[3].xpath('./text()').extract())
#             asnNumHref = response.urljoin(asnNumHref)
#             asnItem = ASNItem(asnNum=asnNum, asnNumHref=asnNumHref, asnName=asnName, ipv4NumIps=ipv4NumIps,
#                               ipv6NumIps=ipv6NumIps, countryCode=response.meta["countryCode"])
#
#             yield asnItem
#
#             yield Request(asnNumHref, meta={"asnNum": asnNum}, callback=self.asn_content_parse, dont_filter=False)
#
#     def asn_content_parse(self, response):
#
#         try:
#             asn_content_list = response.xpath('//div[@class="col-sm-3"]')
#
#             if len(asn_content_list.xpath('./h5/text()').extract()) == 2:
#                 asName = "".join(asn_content_list.xpath('./h5/text()').extract()[0])
#                 orgName = "".join(asn_content_list.xpath('./h5/text()').extract()[1])
#             else:
#                 asName = ""
#                 orgName = "".join(asn_content_list.xpath('./h5/text()').extract()[0])
#             registryRegion = "".join(asn_content_list.xpath('.//a/text()').extract())
#             ipv4Prefixes = "".join(asn_content_list.xpath('./p/text()').extract()[0])
#             ipv6Prefixes = "".join(asn_content_list.xpath('./p/text()').extract()[1])
#             ipv4NumIps = "".join(asn_content_list.xpath('./p/text()').extract()[2])
#             ipv6NumIps = "".join(asn_content_list.xpath('./p/text()').extract()[3])
#
#             cidr_tr_list = response.xpath('//*[@id="pills-ipv4"]//table/tr')
#             for cidr_tr in cidr_tr_list:
#                 td = cidr_tr.xpath('td')
#                 cidr = "".join(td[0].xpath('a/text()').extract())
#                 cidrHref = "".join(td[0].xpath('a/@href').extract())
#                 cidrHref = response.urljoin(cidrHref)
#                 cidrDescription = "".join(td[1].xpath('./text()').extract())
#                 ipNum = "".join(td[2].xpath('./text()').extract())
#
#                 asnContentItem = ASNContentItem(asnNum=response.meta["asnNum"], asName=asName, orgName=orgName,
#                                                 registryRegion=registryRegion, ipv4Prefixes=ipv4Prefixes,
#                                                 ipv6Prefixes=ipv6Prefixes,
#                                                 ipv4NumIps=ipv4NumIps, ipv6NumIps=ipv6NumIps, cidr=cidr,
#                                                 cidrHref=cidrHref, cidrDescription=cidrDescription, ipNum=ipNum)
#                 yield asnContentItem
#         except Exception as e:
#             print("信息不全呗，不处理你了，滚蛋")
